package com.bwda.dsrs.filecenter.service.impl;

import com.bwda.dsrs.base.centerbase.util.PageHelperUtil;
import com.bwda.dsrs.base.util.CopyUtil;
import com.bwda.dsrs.base.util.DateUtil;
import com.bwda.dsrs.base.util.StringUtil;
import com.bwda.dsrs.filecenter.condition.DbScanTableHandleParams;
import com.bwda.dsrs.filecenter.domain.BusinessConstants;
import com.bwda.dsrs.filecenter.domain.condition.dataleveltyperel.DataLevelTypeRelCondition;
import com.bwda.dsrs.filecenter.domain.condition.datasensitivewords.DataSensitiveWordsCondition;
import com.bwda.dsrs.filecenter.domain.condition.datatypewordrel.DataTypeWordRelCondition;
import com.bwda.dsrs.filecenter.domain.condition.dbgrouprel.DbGroupRelCondition;
import com.bwda.dsrs.filecenter.domain.condition.dbinfo.DbInfoCondition;
import com.bwda.dsrs.filecenter.domain.condition.dbscanresult.DbScanResultCondition;
import com.bwda.dsrs.filecenter.domain.condition.scanresulttable.ScanResultTableCondition;
import com.bwda.dsrs.filecenter.domain.po.datalevel.DataLevelPo;
import com.bwda.dsrs.filecenter.domain.po.dataleveltyperel.DataLevelTypeRelPo;
import com.bwda.dsrs.filecenter.domain.po.datasensitivewords.DataSensitiveWordsPo;
import com.bwda.dsrs.filecenter.domain.po.datatype.DataTypePo;
import com.bwda.dsrs.filecenter.domain.po.datatypewordrel.DataTypeWordRelPo;
import com.bwda.dsrs.filecenter.domain.po.dbgroup.DbGroupPo;
import com.bwda.dsrs.filecenter.domain.po.dbgrouprel.DbGroupRelPo;
import com.bwda.dsrs.filecenter.domain.po.dbinfo.DbInfoPo;
import com.bwda.dsrs.filecenter.domain.po.dbinfohistory.DbInfoHistoryPo;
import com.bwda.dsrs.filecenter.domain.po.dbscanhistory.DbScanHistoryPo;
import com.bwda.dsrs.filecenter.domain.po.dbscanresult.DbScanResultPo;
import com.bwda.dsrs.filecenter.domain.po.executionhistory.ExecutionHistoryPo;
import com.bwda.dsrs.filecenter.domain.po.scanresulttable.ScanResultTablePo;
import com.bwda.dsrs.filecenter.domain.po.tableinfohistory.TableInfoHistoryPo;
import com.bwda.dsrs.filecenter.mapper.dsrs.DbScanResultMapper;
import com.bwda.dsrs.filecenter.mapper.dsrs.ScanResultTableMapper;
import com.bwda.dsrs.filecenter.service.*;
import com.bwda.dsrs.filecenter.util.DateTimeUtils;
import com.bwda.dsrs.filecenter.util.SpringUtils;
import com.bwda.dsrs.systemservice.domain.Constants;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import java.sql.Connection;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static com.bwda.dsrs.filecenter.domain.SensRuleConstants.*;

/**
 * @author Administrator
 */
@Service
public class DbScanResultServiceImpl extends BaseServiceImpl<DbScanResultPo, DbScanResultCondition, DbScanResultMapper> implements DbScanResultService {
    private final Logger logger = LoggerFactory.getLogger(DbScanResultServiceImpl.class);
    @Autowired
    private DbScanResultMapper dbScanResultMapper;
    @Autowired
    private DbInfoService dbInfoService;
    @Autowired
    private DbInfoHistoryService dbInfoHistoryService;
    @Autowired
    private DataSensitiveWordsService dataSensitiveWordsService;
    @Autowired
    private AsyncService asyncService;
    @Autowired
    private ScanResultTableMapper scanResultTableMapper;
    @Autowired
    private DbGroupRelService dbGroupRelService;
    @Autowired
    private DbGroupService dbGroupService;
    @Autowired
    private DbScanHistoryService dbScanHistoryService;
    @Autowired
    private DataTypeWordRelService dataTypeWordRelService;
    @Autowired
    private DataTypeService dataTypeService;
    @Autowired
    private DataLevelService dataLevelService;
    @Autowired
    private DataLevelTypeRelService dataLevelTypeRelService;
    @Autowired
    private TableInfoHistoryService tableInfoHistoryService;
    @Autowired
    private ExecutionHistoryService resAuditTaskHistoryService;

    @Value("${percent_match_in_sum}")
    private Integer matchNumPercent;
    @Value("${table_column_scan_flag}")
    private Integer tableColumnScanFlag;

    AtomicInteger index=new AtomicInteger(0);


    @Override
    public void scan(Integer dbId,Integer taskId,Integer taskHistoryId,Integer rowNum){
//        long startTime = System.currentTimeMillis();
//        DbScanResultCondition dbScanResultCondition = new DbScanResultCondition();
//        dbScanResultCondition.setDbId(dbId);
//        dbScanResultMapper.truncate(dbScanResultCondition);
//        scanResultTableMapper.deleteByDbId(dbId);
        //1、获取所有数据库列表和分类分级关联数据
        DbInfoCondition dbInfoCondition = new DbInfoCondition();
        dbInfoCondition.setDbId(dbId);
        dbInfoCondition.setPageSize(Integer.MAX_VALUE);
        dbInfoCondition.setDelFlag(BusinessConstants.NumberConstant.ZERO);
        List<DbInfoPo> dbInfoPoList = dbInfoService.queryList(dbInfoCondition);
        DataSensitiveWordsCondition dataSensitiveWordsCondition = new DataSensitiveWordsCondition();
        dataSensitiveWordsCondition.setStatus(BusinessConstants.NumberConstant.ZERO);
        dataSensitiveWordsCondition.setDelFlag(BusinessConstants.NumberConstant.ZERO);
        dataSensitiveWordsCondition.setPageSize(Integer.MAX_VALUE);
        List<DataSensitiveWordsPo> dataSensitiveWordsPos = dataSensitiveWordsService.queryList(dataSensitiveWordsCondition);
        DataLevelTypeRelCondition dataLevelTypeRelCondition = new DataLevelTypeRelCondition();
        dataLevelTypeRelCondition.setPageSize(Integer.MAX_VALUE);
        List<DataLevelTypeRelPo> dataLevelTypeRelPos = dataLevelTypeRelService.queryList(dataLevelTypeRelCondition);
        List<DataLevelTypeRelPo> dataLevelTypeRelPosSorted = dataLevelTypeRelPos.stream().sorted(Comparator.comparing(DataLevelTypeRelPo::getLevelId).reversed()).collect(Collectors.toList());
        if(!CollectionUtils.isEmpty(dbInfoPoList)){
            for (DbInfoPo db:dbInfoPoList
                 ) {
                IRemoteService iRemoteService = SpringUtils.getBean(db.getDbType().concat("RemoteServiceImpl"));
                try {
                    Connection connection = iRemoteService.login(db);
                    DbScanTableHandleParams.Builder paramsBuilder = DbScanTableHandleParams.newDbScanTableHandleParams()
                            .taskId(taskId)
                            .taskHistoryId(taskHistoryId)
                            .iRemoteService(iRemoteService)
                            .connection(connection)
                            .db(db)
                            .rowNum(rowNum)
                            .dataSensitiveWordsPos(dataSensitiveWordsPos)
                            .dataLevelTypeRelPos(dataLevelTypeRelPosSorted);
                    if(StringUtils.isNotEmpty(db.getDbType()) && "pgsql".equals(db.getDbType())){
                        pgSqlHandle(paramsBuilder.build());
                    }
                    //库级信息统计
//                    iRemoteService.countSequence(connection);
                    //2、获取所有表 这边oracle比较特殊 用户名即库名故传入用户名，对于mysql与sqlserver无影响
                    List<String> tableList = iRemoteService.getAllTableList(connection,db.getNeedScanDb());

                    //统计表数量iRemoteService.countTable(connection)
                    db.setTableNum(tableList.size());
                    //统计视图数量
                    db.setViewNum(iRemoteService.countView(connection));
                    db.setFunctionNum(iRemoteService.countFunction(connection));
                    db.setUpdateTime(new Date());
                    dbInfoService.update(db);
                    //库级信息统计历史入库
                    DbInfoHistoryPo dbInfoHistoryPo = CopyUtil.transfer(db,DbInfoHistoryPo.class);
                    dbInfoHistoryPo.setTaskId(taskId);
                    dbInfoHistoryPo.setTaskHistoryId(taskHistoryId);
                    dbInfoHistoryPo.setCreateTime(new Date());
                    dbInfoHistoryService.insert(dbInfoHistoryPo);
                    //2、获取所有表
                    logger.info("ip{}的{}数据库获取表信息成功",db.getDbIp(),db.getServiceName());
                    //表级别的异步任务
                    for (String tableName:tableList
                         ) {
                        paramsBuilder.tableName(tableName);
                        tableHandle(paramsBuilder.build());
                    }
                } catch (Exception e) {
                    logger.error("ip{}的{}数据库获取表信息失败 error:{}",db.getDbIp(),db.getServiceName(),e);
                }
            }
        }
//        System.err.println("耗时："+(System.currentTimeMillis()-startTime)/1000);
    }
    private void pgSqlHandle(DbScanTableHandleParams dbScanTableHandleParams){
        Integer taskId = dbScanTableHandleParams.getTaskId();
        Integer taskHistoryId = dbScanTableHandleParams.getTaskHistoryId();
        IRemoteService iRemoteService = dbScanTableHandleParams.getIRemoteService();
        DbInfoPo db = dbScanTableHandleParams.getDb();
        Connection connectionPublic = dbScanTableHandleParams.getConnection();
        try {
            //统计表数量
            List<String> dbList = iRemoteService.getAllDBList(connectionPublic,db);
//            connectionPublic.close();
            int tableNum=0,viewNum=0,functionNum=0;
            if(!CollectionUtils.isEmpty(dbList)){
                for (String dbName: dbList) {
                    db.setServiceName(dbName);
                    Connection connection = iRemoteService.login(db);
                    tableNum+=iRemoteService.countTable(connection,null);
                    viewNum+=iRemoteService.countView(connection);
                    functionNum+=iRemoteService.countFunction(connection);
                    //2、获取所有表
                    List<String> tableList = iRemoteService.getAllTableList(connection,null);
                    logger.info("ip{}的{}数据库获取表信息成功",db.getDbIp(),db.getServiceName());
                    dbScanTableHandleParams.setConnection(connection);
                    //表级别的异步任务
                    for (String tableName:tableList
                    ) {
                        DbScanTableHandleParams dbScanTableHandleParams1 = CopyUtil.transfer(dbScanTableHandleParams,DbScanTableHandleParams.class);
                        dbScanTableHandleParams1.setConnection(connection);
                        dbScanTableHandleParams1.setTableName(tableName);
                        tableHandle(dbScanTableHandleParams1);
                    }
                }
                db.setTableNum(tableNum);
                db.setViewNum(viewNum);
                db.setFunctionNum(functionNum);
                db.setUpdateTime(new Date());
                dbInfoService.update(db);
                //库级信息统计历史入库
                DbInfoHistoryPo dbInfoHistoryPo = CopyUtil.transfer(db,DbInfoHistoryPo.class);
                dbInfoHistoryPo.setTaskId(taskId);
                dbInfoHistoryPo.setTaskHistoryId(taskHistoryId);
                dbInfoHistoryPo.setCreateTime(new Date());
                dbInfoHistoryService.insert(dbInfoHistoryPo);
            }
        } catch (Exception e) {
            logger.error("ip{}的{}数据库获取表信息失败 error:{}",db.getDbIp(),db.getServiceName(),e);
        }
    }
    @Override
    public Boolean scanDbGroup(List<Integer> dbGroupIds,Integer taskId,Integer taskHistoryId,Integer rowNum){
        if(!CollectionUtils.isEmpty(dbGroupIds)){
            for (Integer dbGroupId:dbGroupIds
                 ) {
                DbGroupPo dbGroupPo = dbGroupService.query(dbGroupId);
                if(dbGroupPo!=null && dbGroupPo.getDelFlag()!=null && dbGroupPo.getDelFlag().equals(Constants.NOT_DELETED) && dbGroupPo.getStatus()!=null && dbGroupPo.getStatus().equals(Constants.ENABLE)){
                    DbGroupRelCondition dbGroupRelCondition = new DbGroupRelCondition();
                    dbGroupRelCondition.setGroupId(dbGroupId);
                    dbGroupRelCondition.setPageSize(Integer.MAX_VALUE);
                    List<DbGroupRelPo> dbGroupRelPos = dbGroupRelService.queryList(dbGroupRelCondition);
                    if(!CollectionUtils.isEmpty(dbGroupRelPos)){
                        for (DbGroupRelPo dbGroupRelPo:dbGroupRelPos
                        ) {
                            asyncService.asyncScan(dbGroupRelPo.getDbId(),taskId,taskHistoryId,rowNum);
                        }
                    }
                }
            }
        }
        return true;
    }

    @Override
    public void tableHandle(DbScanTableHandleParams dbScanTableHandleParams){
        List<DbScanResultPo> dbScanResultPos = new ArrayList<>();
//        List<Object> data = new ArrayList<>();
        List<Map<String,Object>> data = new ArrayList<>();
        Integer taskId = dbScanTableHandleParams.getTaskId();
        Integer taskHistoryId = dbScanTableHandleParams.getTaskHistoryId();
        IRemoteService iRemoteService = dbScanTableHandleParams.getIRemoteService();
        Connection connection = dbScanTableHandleParams.getConnection();
        String tableName = dbScanTableHandleParams.getTableName();
        Integer rowNum = dbScanTableHandleParams.getRowNum();
        int sumRowNum = 0;
        DbInfoPo db = dbScanTableHandleParams.getDb();
//        Integer typeDependentNum = dbScanTableHandleParams.getTypeDependentNum();
        List<DataSensitiveWordsPo> dataSensitiveWordsPos = dbScanTableHandleParams.getDataSensitiveWordsPos();
//        List<DataLevelTypeRelPo> dataLevelTypeRelPos = dbScanTableHandleParams.getDataLevelTypeRelPos();
        //3、获取所有列和列数据
        try {
            dbScanResultPos = iRemoteService.getTableColumns(connection,tableName);
            logger.info("ip{}的{}数据库{}表获取列信息成功",db.getDbIp(),db.getServiceName(),tableName);
        } catch (Exception e) {
            logger.error("ip{}的{}数据库{}表获取列信息失败 error:{}",db.getDbIp(),db.getServiceName(),tableName,e);
        }
        if(!CollectionUtils.isEmpty(dbScanResultPos)){
            try {
                sumRowNum = iRemoteService.countRowNum(connection,dbScanResultPos.get(0).getTableName());
                dbScanTableHandleParams.setSumRowNum(sumRowNum);
                logger.info("ip{}的{}数据库{}表获取数据行数成功",db.getDbIp(),db.getServiceName(),dbScanResultPos.get(0).getTableName());
            } catch (Exception e) {
                logger.error("ip{}的{}数据库{}表获取数据行数失败 error:{}",db.getDbIp(),db.getServiceName(),dbScanResultPos.get(0).getTableName(),e);
            }
            if(tableColumnScanFlag.equals(BusinessConstants.INTEGER_ONE)){
                try {
                    data = iRemoteService.getData(connection,tableName,null,rowNum,sumRowNum);
                    logger.info("ip{}的{}数据库{}表获取数据成功",db.getDbIp(),db.getServiceName(),tableName);
                } catch (Exception e) {
                    logger.error("ip{}的{}数据库{}表获取数据失败 error:{}",db.getDbIp(),db.getServiceName(),tableName,e);
                }
            }
            for (DbScanResultPo dbScanResultPo:dbScanResultPos
                    ) {
                dbScanResultPo.setDbId(db.getDbId());
                dbScanResultPo.setDbType(db.getDbType());
                if(tableColumnScanFlag.equals(BusinessConstants.TWO_INT)){
                    try {
                        data = iRemoteService.getData(connection,tableName,dbScanResultPo.getColumnName(),rowNum,sumRowNum);
                        logger.info("ip{}的{}数据库{}表{}列获取数据成功",db.getDbIp(),db.getServiceName(),tableName,dbScanResultPo.getColumnName());
                    } catch (Exception e) {
                        logger.error("ip{}的{}数据库{}表{}列获取数据失败 error:{}",db.getDbIp(),db.getServiceName(),tableName,dbScanResultPo.getColumnName(),e);
                    }
                }
                if(!CollectionUtils.isEmpty(data)){
                    //4、匹配处理
//                Integer[] matchResult = match(dataSensitiveWordsPos,data,dbScanResultPo.getColumnName());
                    String[] matchResult = match(dataSensitiveWordsPos,data,dbScanResultPo.getColumnName(),dbScanResultPo.getColumnRemark());
                    if(StringUtils.isNotEmpty(matchResult[0])){dbScanResultPo.setWordIds(matchResult[0]);}
                    if(StringUtils.isNotEmpty(matchResult[1])){dbScanResultPo.setMatchNums(Integer.valueOf(matchResult[1]));}
                    if(StringUtils.isNotEmpty(matchResult[2])){dbScanResultPo.setScanNums(Integer.valueOf(matchResult[2]));}
                    //截取字符串，防止插入数据出错。后期优化去掉
                    if(StringUtils.isNotEmpty(matchResult[3])){
                        if(matchResult[3].length() > 200){
                            dbScanResultPo.setRemark(matchResult[3].substring(0,200));
                        }else {
                            dbScanResultPo.setRemark(matchResult[3]);
                        }

                    }
                }
                //5、列匹配信息入库
                DbScanResultCondition dbScanResultCondition = new DbScanResultCondition();
                dbScanResultCondition.setDbId(db.getDbId());
                dbScanResultCondition.setTableName(dbScanResultPo.getTableName());
                dbScanResultCondition.setColumnName(dbScanResultPo.getColumnName());
                dbScanResultCondition.setDelFlag(0);
                List<DbScanResultPo> dbScanResultPos1 = this.getMapper().selectList(dbScanResultCondition);
                Date date = new Date();
                dbScanResultPo.setUpdateTime(date);
                //增量更新，这边大概是0.02秒
                if(CollectionUtils.isEmpty(dbScanResultPos1)){
                    dbScanResultPo.setCreateTime(date);
                    this.getMapper().insert(dbScanResultPo);
                }
                else {
                    dbScanResultPo.setScanResultId(dbScanResultPos1.get(0).getScanResultId());
                    this.getMapper().update(dbScanResultPo);
                }
                //列匹配信息历史入库，这边大概是0.02秒
                DbScanHistoryPo dbScanHistoryPo = CopyUtil.transfer(dbScanResultPo,DbScanHistoryPo.class);
                dbScanHistoryPo.setTaskId(taskId);
                dbScanHistoryPo.setTaskHistoryId(taskHistoryId);
                dbScanHistoryPo.setSubTaskHistoryId(taskHistoryId);
                dbScanHistoryPo.setCreateTime(date);
                dbScanHistoryService.insert(dbScanHistoryPo);
            }
            //逻辑删除所有不存在的列
            DbScanResultCondition dbScanResultCondition = new DbScanResultCondition();
            dbScanResultCondition.setDbId(db.getDbId());
            dbScanResultCondition.setTableName(tableName);
            dbScanResultCondition.setDelFlag(0);
            List<DbScanResultPo> dbScanResultPosOld = this.getMapper().selectList(dbScanResultCondition);
            List<String> columnNames = dbScanResultPos.stream().map(p->p.getColumnName()).collect(Collectors.toList());
            if(!CollectionUtils.isEmpty(dbScanResultPosOld)){
                for (DbScanResultPo dbScanResultPo:dbScanResultPosOld
                     ) {
                    if(!columnNames.contains(dbScanResultPo.getColumnName())){
                        this.getMapper().delete(dbScanResultPo.getScanResultId());
                    }
                }
            }
            //6、关联分类分级汇聚到表级数据
            analyzeTypeAndLevel(dbScanResultPos,dbScanTableHandleParams);
            logger.info("--匹配结束");
        }
    }
    private void analyzeTypeAndLevel(List<DbScanResultPo> dbScanResultPos,DbScanTableHandleParams dbScanTableHandleParams){
        logger.info("分类分级统计开始...");
        DbInfoPo db = dbScanTableHandleParams.getDb();
        Integer taskId = dbScanTableHandleParams.getTaskId();
        Integer taskHistoryId = dbScanTableHandleParams.getTaskHistoryId();
        Integer typeDependentNum = dbScanTableHandleParams.getTypeDependentNum();
        List<DataSensitiveWordsPo> dataSensitiveWordsPos = dbScanTableHandleParams.getDataSensitiveWordsPos();
        List<DataLevelTypeRelPo> dataLevelTypeRelPosSorted = dbScanTableHandleParams.getDataLevelTypeRelPos();
        ScanResultTablePo scanResultTablePo = new ScanResultTablePo();
        scanResultTablePo.setDbId(db.getDbId());
        scanResultTablePo.setDbType(db.getDbType());
        scanResultTablePo.setTableRemark(dbScanResultPos.get(0).getTableRemark());
        scanResultTablePo.setTableName(dbScanResultPos.get(0).getTableName());
        List<Integer> wordsIds = dbScanResultPos.stream().map(p->Integer.valueOf(StringUtils.isNotBlank(p.getWordIds())?p.getWordIds():"0")).filter(p->p>0).distinct().collect(Collectors.toList())  ;
        if(!CollectionUtils.isEmpty(wordsIds)){
            scanResultTablePo.setSensLabel(getNamesByWordIds(dataSensitiveWordsPos,wordsIds));
            //敏感等级按照倒序排列匹配，就高不就低原则
            for (DataLevelTypeRelPo dataLevelTypeRelPo:dataLevelTypeRelPosSorted
            ) {
                DataTypeWordRelCondition dataTypeWordRelCondition = new DataTypeWordRelCondition();
                dataTypeWordRelCondition.setTypeId(dataLevelTypeRelPo.getTypeId());
                dataTypeWordRelCondition.setPageSize(Integer.MAX_VALUE);
                List<DataTypeWordRelPo> dataTypeWordRelPos = dataTypeWordRelService.queryList(dataTypeWordRelCondition);
                if(!CollectionUtils.isEmpty(dataTypeWordRelPos)){
                    int num = 0;
                    for (DataTypeWordRelPo dataTypeWordRelPo:dataTypeWordRelPos
                    ) {
                        if(wordsIds.contains(dataTypeWordRelPo.getWordId())){
                            num++;
                        }
                        if(num>=typeDependentNum){
                            //分类分析结果，定义常量n(默认为2)，出现n个敏感字段才认为是A1-1类。即同时出现姓名+手机才认为是自然人。
                            DataTypePo dataTypePo = dataTypeService.query(dataTypeWordRelPo.getTypeId());
                            scanResultTablePo.setTypeCode(dataTypePo.getTypeCode());
                            scanResultTablePo.setTypeName(dataTypePo.getTypeName());
                            DataLevelPo dataLevelPo = dataLevelService.query(dataLevelTypeRelPo.getLevelId());
                            scanResultTablePo.setLevel(dataLevelPo.getLevel());
                            scanResultTablePo.setLevelName(dataLevelPo.getName());
                            break;
                        }
                    }
                }
            }
//                List<DataTypePo> dataTypePos = getDataTypePoByTypeIds(getDataTypeWordRelPoByWordIds(wordsIds));
//                if(!CollectionUtils.isEmpty(dataTypePos)){
//                    scanResultTableVo.setTypeCode(dataTypePos.get(0).getTypeCode());
//                    scanResultTableVo.setTypeName(dataTypePos.get(0).getTypeName());
////                    scanResultTableVo.setTypeCode(dataTypePos.stream().map(p->p.getTypeCode()).distinct().collect(Collectors.joining(",")));
////                    scanResultTableVo.setTypeName(dataTypePos.stream().map(p->p.getTypeName()).distinct().collect(Collectors.joining(",")));
//                }
//                List<DataLevelPo> dataLevelPos = getDataLevelPoByLevelIds(getDataLevelTypeRelPoByTypeIs(dataTypePos.stream().map((p->p.getTypeId())).distinct().collect(Collectors.toList())));
//                if(!CollectionUtils.isEmpty(dataLevelPos)){
//                    Optional<DataLevelPo> dataLevelPo = dataLevelPos.stream().max(Comparator.comparingInt(DataLevelPo::getLevelId));
//                    scanResultTableVo.setLevelName(dataLevelPo.get().getName());
////                    scanResultTableVo.setLevelName(dataLevelPos.stream().map(p->p.getName()).distinct().collect(Collectors.joining(",")));
//                }
        }
        scanResultTablePo.setRemark("");
        scanResultTablePo.setDelFlag(0);
        //行数与列数统计
//        try {
//            scanResultTablePo.setRowNum(iRemoteService.countRowNum(connection,dbScanResultPos.get(0).getTableName()));
//            logger.info("ip{}的{}数据库{}表获取数据行数成功",db.getDbIp(),db.getServiceName(),dbScanResultPos.get(0).getTableName());
//        } catch (Exception e) {
//            logger.error("ip{}的{}数据库{}表获取数据行数失败 error:{}",db.getDbIp(),db.getServiceName(),dbScanResultPos.get(0).getTableName(),e);
//        }
        scanResultTablePo.setRowNum(dbScanTableHandleParams.getSumRowNum());
        scanResultTablePo.setColumnNum(dbScanResultPos.stream().collect(Collectors.counting()).intValue());
        //表级数据入库
        ScanResultTableCondition scanResultTableCondition = new ScanResultTableCondition();
        scanResultTableCondition.setDbId(db.getDbId());
        scanResultTableCondition.setTableName(scanResultTablePo.getTableName());
        scanResultTableCondition.setDelFlag(0);
        List<ScanResultTablePo> scanResultTablePos = scanResultTableMapper.selectList(scanResultTableCondition);
        Date date = new Date();
        scanResultTablePo.setUpdateTime(date);
        //增量更新
        if(CollectionUtils.isEmpty(scanResultTablePos)){
            scanResultTablePo.setCreateTime(date);
            scanResultTableMapper.insert(scanResultTablePo);
        }
        else {
            scanResultTablePo.setId(scanResultTablePos.get(0).getId());
            scanResultTableMapper.update(scanResultTablePo);
        }
        //表级数据历史入库
        TableInfoHistoryPo tableInfoHistoryPo = CopyUtil.transfer(scanResultTablePo, TableInfoHistoryPo.class);
        tableInfoHistoryPo.setTaskId(taskId);
        tableInfoHistoryPo.setTaskHistoryId(taskHistoryId);
        tableInfoHistoryPo.setCreateTime(date);
        tableInfoHistoryService.insert(tableInfoHistoryPo);
        //获取所有表数量
        int resourceSize = db.getTableNum();


        ExecutionHistoryPo historyPo = resAuditTaskHistoryService.query(taskHistoryId);
        if(taskHistoryId > 0 && historyPo != null){

            logger.info("resourceSize====" + resourceSize+"====index====="+index);
            //计算进度百分比
            Long process = Math.round((double) index.incrementAndGet()/ (double) resourceSize * 100);
            //当前执行时长
            Long tmpExeDur = DateTimeUtils.minusDate(historyPo.getBeginTime(), DateTimeUtils.getNowDate());
            //更新任务执行进度
            resAuditTaskHistoryService.updateTaskHistoryProcess(taskHistoryId.longValue(), process, tmpExeDur);
            if(index.get() == resourceSize || process >=100){
                historyPo.setExecutionStatus(Constants.ResAuditTaskExeStatus.TES_EXECUTED.intValue());
                historyPo.setSpeedProgress(Constants.ResAuditTaskExeResult.ER_FULL_PROCESS.intValue());
                historyPo.setEndTime(DateTimeUtils.getNowDate());
                resAuditTaskHistoryService.update(historyPo);
            }
        }

        logger.info("分类分级统计结束");
    }

    @Override
    public void updateTypeAndLevel(List<DbScanResultPo> dbScanResultPos,DbScanTableHandleParams dbScanTableHandleParams){
        logger.info("ip{}的{}表分类分级更新开始",dbScanTableHandleParams.getDb().getDbIp(),dbScanResultPos.get(0).getTableName());
        DbInfoPo db = dbScanTableHandleParams.getDb();
        Integer typeDependentNum = dbScanTableHandleParams.getTypeDependentNum();
        List<DataSensitiveWordsPo> dataSensitiveWordsPos = dbScanTableHandleParams.getDataSensitiveWordsPos();
        List<DataLevelTypeRelPo> dataLevelTypeRelPosSorted = dbScanTableHandleParams.getDataLevelTypeRelPos();
        ScanResultTableCondition scanResultTableCondition = new ScanResultTableCondition();
        scanResultTableCondition.setDbId(db.getDbId());
        scanResultTableCondition.setTableName(dbScanResultPos.get(0).getTableName());
        scanResultTableCondition.setDelFlag(0);
        List<ScanResultTablePo> scanResultTablePos = scanResultTableMapper.selectList(scanResultTableCondition);
        ScanResultTablePo scanResultTablePo;
        if(CollectionUtils.isEmpty(scanResultTablePos)){
            scanResultTablePo = new ScanResultTablePo();
            scanResultTablePo.setDbId(db.getDbId());
            scanResultTablePo.setDbType(db.getDbType());
            scanResultTablePo.setTableRemark(dbScanResultPos.get(0).getTableRemark());
            scanResultTablePo.setTableName(dbScanResultPos.get(0).getTableName());
            scanResultTablePo.setRemark("");
            scanResultTablePo.setDelFlag(0);
            scanResultTablePo.setColumnNum(dbScanResultPos.stream().collect(Collectors.counting()).intValue());
            logger.error("ip{}的{}表未找到分类分级统计数据，无法更新,将插入一条数据，但无法保证准确性",dbScanTableHandleParams.getDb().getDbIp(),dbScanResultPos.get(0).getTableName());
        }
        else {
            scanResultTablePo = scanResultTablePos.get(0);
        }
        List<Integer> wordsIds = dbScanResultPos.stream().map(p->Integer.valueOf(StringUtils.isNotBlank(p.getWordIds())?p.getWordIds():"0")).filter(p->p>0).distinct().collect(Collectors.toList())  ;
        if(!CollectionUtils.isEmpty(wordsIds)){
            scanResultTablePo.setSensLabel(getNamesByWordIds(dataSensitiveWordsPos,wordsIds));
            //敏感等级按照倒序排列匹配，就高不就低原则
            for (DataLevelTypeRelPo dataLevelTypeRelPo:dataLevelTypeRelPosSorted
            ) {
                DataTypeWordRelCondition dataTypeWordRelCondition = new DataTypeWordRelCondition();
                dataTypeWordRelCondition.setTypeId(dataLevelTypeRelPo.getTypeId());
                dataTypeWordRelCondition.setPageSize(Integer.MAX_VALUE);
                List<DataTypeWordRelPo> dataTypeWordRelPos = dataTypeWordRelService.queryList(dataTypeWordRelCondition);
                if(!CollectionUtils.isEmpty(dataTypeWordRelPos)){
                    int num = 0;
                    for (DataTypeWordRelPo dataTypeWordRelPo:dataTypeWordRelPos
                    ) {
                        if(wordsIds.contains(dataTypeWordRelPo.getWordId())){
                            num++;
                        }
                        if(num>=typeDependentNum){
                            //分类分析结果，定义常量n(默认为2)，出现n个敏感字段才认为是A1-1类。即同时出现姓名+手机才认为是自然人。
                            DataTypePo dataTypePo = dataTypeService.query(dataTypeWordRelPo.getTypeId());
                            scanResultTablePo.setTypeCode(dataTypePo.getTypeCode());
                            scanResultTablePo.setTypeName(dataTypePo.getTypeName());
                            DataLevelPo dataLevelPo = dataLevelService.query(dataLevelTypeRelPo.getLevelId());
                            scanResultTablePo.setLevel(dataLevelPo.getLevel());
                            scanResultTablePo.setLevelName(dataLevelPo.getName());
                            break;
                        }
                    }
                }
            }
        }
        //增量更新
        Date date = new Date();
        scanResultTablePo.setUpdateTime(date);
        if(CollectionUtils.isEmpty(scanResultTablePos)){
            scanResultTablePo.setCreateTime(date);
            scanResultTableMapper.insert(scanResultTablePo);
        }
        else {
            //更新
            scanResultTableMapper.update(scanResultTablePo);
            logger.info("分类分级更新结束");
        }

    }

    /**
     * 根据数据库敏感规则rex匹配（旧）
     * @param dataSensitiveWordsPos
     * @param data
     * @param columnName
     * @return
     */
    private Integer[] match(List<DataSensitiveWordsPo> dataSensitiveWordsPos,List<Map<String,Object>> data,String columnName) {
        Integer wordId = 0,scanNum=0;
        Integer[] result = new Integer[3];
        List<Integer> words = new ArrayList<>();
        for (Map<String,Object> map:data
                ) {
            Object value = map.get(columnName);
            if(value != null){
                //空不计算总次数
                for (DataSensitiveWordsPo dataSensitiveWordsPo : dataSensitiveWordsPos) {
                    Pattern pattern = Pattern.compile(dataSensitiveWordsPo.getSensitiveContent());
                    Matcher matcher = pattern.matcher(value.toString());
                    while (matcher.find()) {
                        wordId = dataSensitiveWordsPo.getWordId();
                        break;
                    }
                }
                if(wordId>0){
                    words.add(wordId);
                }
                scanNum++;
            }
        }
        result[2]=scanNum;
        //计算最大匹配次数
        if(!CollectionUtils.isEmpty(words)){
            Map<Integer,Long> resultGroupBy = words.stream().collect(Collectors.groupingBy(Function.identity() ,Collectors.counting()));
            Optional<Map.Entry<Integer, Long>> resultMax = resultGroupBy.entrySet().stream().max(Map.Entry.comparingByValue());
            result[0]=resultMax.get().getKey();
            result[1]=resultMax.get().getValue().intValue();
        }
        return result;
    }

    /**
     * 新规则匹配
     * @param data
     * @param columnName
     * @param columnRemark
     * @return
     */
    private String[] match(List<DataSensitiveWordsPo> dataSensitiveWordsPos,List<Map<String,Object>> data,String columnName,String columnRemark) {
        logger.info("{}列数据匹配开始...",columnName);
        int scanNum=0,sampleNum=3;
        String[] result = new String[4];//result[0]WordIds,result[1]MatchNums,result[0]setScanNums,result[0]sampleData
        List<Integer> words = new ArrayList<>();
        Map<Integer,String> sampleData = new HashMap<>(16);
        for (Map<String,Object> map:data
                ) {
            Object value = map.get(columnName);
            if(value != null && StringUtils.isNotBlank(value.toString())){
                String valueString = value.toString();
                int valueLength = valueString.length();
                //空不计算总次数
                scanNum++;
                for (DataSensitiveWordsPo d:dataSensitiveWordsPos
                     ) {
//                    boolean columnNameExFlag=true,columnRemarkExFlag=true,
//                            lengthFlag=true,maxLengthFlag=true,minLengthFlag=true,
//                            columnNameInFlag=true,columnRemarkInFlag=true,contentEnumFlag=true,contentEndFlag=true,rexFlag=true;
                    if(d.getMinLength()==null && d.getMaxLength()==null && StringUtils.isBlank(d.getLength())
                            && StringUtils.isBlank(d.getColumnNameEx()) && StringUtils.isBlank(d.getColumnRemarkEx())
                            && StringUtils.isBlank(d.getContentEnum()) && StringUtils.isBlank(d.getContentEnd())
                            && StringUtils.isBlank(d.getColumnNameIn()) && StringUtils.isBlank(d.getColumnRemarkIn())
                            && StringUtils.isBlank(d.getSensitiveContent()) && StringUtils.isBlank(d.getSensitiveRex1())
                            && StringUtils.isBlank(d.getSensitiveRex2())&& StringUtils.isBlank(d.getSensitiveRex3())
                            && StringUtils.isBlank(d.getSensitiveRex4())&& StringUtils.isBlank(d.getSensitiveRex5())
                    ){
                        continue;
                    }
                    boolean columnNameInFlag=false,columnRemarkInFlag=false,sensitiveContentFlag=false,rexFlag1=false,rexFlag2=false,rexFlag3=false,rexFlag4=false,rexFlag5=false;
                    if(StringUtils.isNotBlank(d.getColumnNameEx()) && matchString(d.getColumnNameEx(),columnName,"equal")){
                        continue;
                    }
                    if(StringUtils.isNotBlank(d.getColumnRemarkEx()) && StringUtils.isNotBlank(columnRemark) && matchString(d.getColumnRemarkEx(),columnRemark.toLowerCase(),"equal")){
                        continue;
                    }
                    if(StringUtils.isNotBlank(d.getLength()) && !matchLength(d.getLength(),valueString.length())){
                        continue;
                    }
                    if(d.getMaxLength()!=null && d.getMaxLength()<valueLength){
                        continue;
                    }
                    if(d.getMinLength()!=null && d.getMinLength()>valueLength){
                        continue;
                    }
                    if(StringUtils.isNotBlank(d.getContentEnum()) && !matchString(d.getContentEnum(),valueString,"contains")){
                        continue;
                    }
                    if(StringUtils.isNotBlank(d.getContentEnd()) && !matchString(d.getContentEnd(),valueString,"endsWith")){
                        continue;
                    }
                    if(StringUtils.isBlank(d.getColumnNameIn()) && StringUtils.isBlank(d.getColumnRemarkIn())){
                        columnNameInFlag = true;
                        columnRemarkInFlag = true;
                    }
                    if(StringUtils.isNotBlank(d.getColumnNameIn()) && matchString(d.getColumnNameIn(),columnName,"contains")){
                        columnNameInFlag = true;
                    }
                    if(StringUtils.isNotBlank(d.getColumnRemarkIn()) && StringUtils.isNotBlank(columnRemark) && matchString(d.getColumnRemarkIn(),columnRemark.toLowerCase(),"contains")){
                        columnRemarkInFlag = true;
                    }
                    if(StringUtils.isBlank(d.getSensitiveContent()) && StringUtils.isBlank(d.getSensitiveRex1())
                            && StringUtils.isBlank(d.getSensitiveRex2()) && StringUtils.isBlank(d.getSensitiveRex3())
                    && StringUtils.isBlank(d.getSensitiveRex4()) && StringUtils.isBlank(d.getSensitiveRex5())){
                        sensitiveContentFlag = true;
                        rexFlag1 = true;
                        rexFlag2 = true;
                        rexFlag3 = true;
                        rexFlag4 = true;
                        rexFlag5 = true;
                    }
                    if(StringUtils.isNotBlank(d.getSensitiveContent()) && matchRex(d.getSensitiveContent(),valueString)){
                        sensitiveContentFlag = true;
                    }
                    if(StringUtils.isNotBlank(d.getSensitiveRex1()) && matchRex(d.getSensitiveRex1(),valueString)){
                        rexFlag1 = true;
                    }
                    if(StringUtils.isNotBlank(d.getSensitiveRex2()) && matchRex(d.getSensitiveRex2(),valueString)){
                        rexFlag2 = true;
                    }
                    if(StringUtils.isNotBlank(d.getSensitiveRex3()) && matchRex(d.getSensitiveRex3(),valueString)){
                        rexFlag3 = true;
                    }
                    if(StringUtils.isNotBlank(d.getSensitiveRex4()) && matchRex(d.getSensitiveRex4(),valueString)){
                        rexFlag4 = true;
                    }
                    if(StringUtils.isNotBlank(d.getSensitiveRex5()) && matchRex(d.getSensitiveRex5(),valueString)){
                        rexFlag5 = true;
                    }
                    if((columnNameInFlag||columnRemarkInFlag)&&(sensitiveContentFlag||rexFlag1||rexFlag2||rexFlag3||rexFlag4||rexFlag5)){
                        words.add(d.getWordId());
                        if(StringUtils.isEmpty(sampleData.get(d.getWordId()))){
                            sampleData.put(d.getWordId(),value.toString());
                        }else {
                            if(sampleData.get(d.getWordId()).split(",").length<sampleNum) {
                                sampleData.put(d.getWordId(),sampleData.get(d.getWordId()).concat(",").concat(value.toString()));
                            }
                        }
                    }
                }
//
//                if(isTelephoneNum(value.toString())){
//                    words.add(TELEPHONENUM_ID);
//                    if(StringUtils.isEmpty(sampleData.get(TELEPHONENUM_ID))){
//                        sampleData.put(TELEPHONENUM_ID,value.toString());
//                    }else {
//                        if(sampleData.get(TELEPHONENUM_ID).split(",").length<sampleNum)
//                        sampleData.put(TELEPHONENUM_ID,sampleData.get(TELEPHONENUM_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isPhoneNum(value.toString())){
//                    words.add(PHONENUM_ID);
//                    if(StringUtils.isEmpty(sampleData.get(PHONENUM_ID))){
//                        sampleData.put(PHONENUM_ID,value.toString());
//                    }else {
//                        if(sampleData.get(PHONENUM_ID).split(",").length<sampleNum)
//                            sampleData.put(PHONENUM_ID,sampleData.get(PHONENUM_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isDriveLicence(value.toString(),columnName,columnRemark)){
//                    words.add(DRIVELICENCE_ID);
//                    if(StringUtils.isEmpty(sampleData.get(DRIVELICENCE_ID))){
//                        sampleData.put(DRIVELICENCE_ID,value.toString());
//                    }else {
//                        if(sampleData.get(DRIVELICENCE_ID).split(",").length<sampleNum)
//                            sampleData.put(DRIVELICENCE_ID,sampleData.get(DRIVELICENCE_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isBankAccount(value.toString())){
//                    words.add(BANKACCOUNT_ID);
//                    if(StringUtils.isEmpty(sampleData.get(BANKACCOUNT_ID))){
//                        sampleData.put(BANKACCOUNT_ID,value.toString());
//                    }else {
//                        if(sampleData.get(BANKACCOUNT_ID).split(",").length<sampleNum)
//                            sampleData.put(BANKACCOUNT_ID,sampleData.get(BANKACCOUNT_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isEmail(value.toString())){
//                    words.add(EMAIL_ID);
//                    if(StringUtils.isEmpty(sampleData.get(EMAIL_ID))){
//                        sampleData.put(EMAIL_ID,value.toString());
//                    }else {
//                        if(sampleData.get(EMAIL_ID).split(",").length<sampleNum)
//                            sampleData.put(EMAIL_ID,sampleData.get(EMAIL_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isQQNum(value.toString(),columnName,columnRemark)){
//                    words.add(QQNUM_ID);
//                    if(StringUtils.isEmpty(sampleData.get(QQNUM_ID))){
//                        sampleData.put(QQNUM_ID,value.toString());
//                    }else {
//                        if(sampleData.get(QQNUM_ID).split(",").length<sampleNum)
//                            sampleData.put(QQNUM_ID,sampleData.get(QQNUM_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isOccupation(value.toString())){
//                    words.add(OCCUPATION_ID);
//                    if(StringUtils.isEmpty(sampleData.get(OCCUPATION_ID))){
//                        sampleData.put(OCCUPATION_ID,value.toString());
//                    }else {
//                        if(sampleData.get(OCCUPATION_ID).split(",").length<sampleNum)
//                            sampleData.put(OCCUPATION_ID,sampleData.get(OCCUPATION_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isWorkPlace(value.toString())){
//                    words.add(WORKPLACE_ID);
//                    if(StringUtils.isEmpty(sampleData.get(WORKPLACE_ID))){
//                        sampleData.put(WORKPLACE_ID,value.toString());
//                    }else {
//                        if(sampleData.get(WORKPLACE_ID).split(",").length<sampleNum)
//                            sampleData.put(WORKPLACE_ID,sampleData.get(WORKPLACE_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isBirthday(value.toString(),columnName,columnRemark)){
//                    words.add(BIRTHDAY_ID);
//                    if(StringUtils.isEmpty(sampleData.get(BIRTHDAY_ID))){
//                        sampleData.put(BIRTHDAY_ID,value.toString());
//                    }else {
//                        if(sampleData.get(BIRTHDAY_ID).split(",").length<sampleNum)
//                            sampleData.put(BIRTHDAY_ID,sampleData.get(BIRTHDAY_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isGender(value.toString(),columnName,columnRemark)){
//                    words.add(GENDER_ID);
//                    if(StringUtils.isEmpty(sampleData.get(GENDER_ID))){
//                        sampleData.put(GENDER_ID,value.toString());
//                    }else {
//                        if(sampleData.get(GENDER_ID).split(",").length<sampleNum)
//                            sampleData.put(GENDER_ID,sampleData.get(GENDER_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isHometown(value.toString())){
//                    words.add(HOMETOWN_ID);
//                    if(StringUtils.isEmpty(sampleData.get(HOMETOWN_ID))){
//                        sampleData.put(HOMETOWN_ID,value.toString());
//                    }else {
//                        if(sampleData.get(HOMETOWN_ID).split(",").length<sampleNum)
//                            sampleData.put(HOMETOWN_ID,sampleData.get(HOMETOWN_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isPassword(value.toString(),columnName,columnRemark)){
//                    words.add(PASSWORD_ID);
//                    if(StringUtils.isEmpty(sampleData.get(PASSWORD_ID))){
//                        sampleData.put(PASSWORD_ID,value.toString());
//                    }else {
//                        if(sampleData.get(PASSWORD_ID).split(",").length<sampleNum)
//                            sampleData.put(PASSWORD_ID,sampleData.get(PASSWORD_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isMac(value.toString())){
//                    words.add(MAC_ID);
//                    if(StringUtils.isEmpty(sampleData.get(MAC_ID))){
//                        sampleData.put(MAC_ID,value.toString());
//                    }else {
//                        if(sampleData.get(MAC_ID).split(",").length<sampleNum)
//                            sampleData.put(MAC_ID,sampleData.get(MAC_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isOs(value.toString())){
//                    words.add(OS_ID);
//                    if(StringUtils.isEmpty(sampleData.get(OS_ID))){
//                        sampleData.put(OS_ID,value.toString());
//                    }else {
//                        if(sampleData.get(OS_ID).split(",").length<sampleNum)
//                            sampleData.put(OS_ID,sampleData.get(OS_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isName(value.toString())){
//                    words.add(NAME_ID);
//                    if(StringUtils.isEmpty(sampleData.get(NAME_ID))){
//                        sampleData.put(NAME_ID,value.toString());
//                    }else {
//                        if(sampleData.get(NAME_ID).split(",").length<sampleNum)
//                            sampleData.put(NAME_ID,sampleData.get(NAME_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isIdNum(value.toString(),columnName,columnRemark)){
//                    words.add(IDNUM_ID);
//                    if(StringUtils.isEmpty(sampleData.get(IDNUM_ID))){
//                        sampleData.put(IDNUM_ID,value.toString());
//                    }else {
//                        if(sampleData.get(IDNUM_ID).split(",").length<sampleNum)
//                            sampleData.put(IDNUM_ID,sampleData.get(IDNUM_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
//                if(isAddress(value.toString())){
//                    words.add(ADDRESS_ID);
//                    if(StringUtils.isEmpty(sampleData.get(ADDRESS_ID))){
//                        sampleData.put(ADDRESS_ID,value.toString());
//                    }else {
//                        if(sampleData.get(ADDRESS_ID).split(",").length<sampleNum)
//                            sampleData.put(ADDRESS_ID,sampleData.get(ADDRESS_ID).concat(",").concat(value.toString()));
//                    }
//                    continue;
//                }
                if(StringUtils.isEmpty(sampleData.get(0))){
                    sampleData.put(0,value.toString());
                }else {
                    if(sampleData.get(0).split(",").length<sampleNum) {
                        sampleData.put(0,sampleData.get(0).concat(",").concat(value.toString()));
                    }
                }
            }
        }
        result[2]=String.valueOf(scanNum);
        result[3]=sampleData.get(0);
        //计算最大匹配次数
        if(!CollectionUtils.isEmpty(words)){
            Map<Integer,Long> resultGroupBy = words.stream().collect(Collectors.groupingBy(Function.identity() ,Collectors.counting()));
            Optional<Map.Entry<Integer, Long>> resultMax = resultGroupBy.entrySet().stream().max(Map.Entry.comparingByValue());
            double percent = 100*resultMax.get().getValue().intValue()/scanNum;
            if(percent>matchNumPercent){
                result[0]=String.valueOf(resultMax.get().getKey());
                result[1]=String.valueOf(resultMax.get().getValue().intValue());
                result[3]=sampleData.get(resultMax.get().getKey());
            }
        }
        logger.info("{}列数据匹配结束",columnName);
        return result;
    }

    /**
     * 遍历表，查匹配的敏感字段
     * @param dbId
     * @param tableName
     * @return
     */
    private List<Integer> getWordsByTableName(Integer dbId,String tableName) {
        DbScanResultCondition dbScanResultCondition = new DbScanResultCondition();
        dbScanResultCondition.setDbId(dbId);
        dbScanResultCondition.setTableName(tableName);
        List<DbScanResultPo> dbScanResultPos1 = this.queryTableeColumnList(dbScanResultCondition);
        List<Integer> wordIds  = new ArrayList<>();
        if (!CollectionUtils.isEmpty(dbScanResultPos1)) {
            for (DbScanResultPo dbScanResultPo : dbScanResultPos1) {
                if(StringUtils.isNotBlank(dbScanResultPo.getWordIds())){
                    wordIds.add(Integer.parseInt(dbScanResultPo.getWordIds()));
                }
            }
        }
        return wordIds.stream().distinct().collect(Collectors.toList());
    }

    /**
     * 匹配敏感标签
     * @param dataSensitiveWordsPos
     * @param wordIds
     * @return
     */
    private String getNamesByWordIds(List<DataSensitiveWordsPo> dataSensitiveWordsPos,List<Integer> wordIds){
        List<String > names = new ArrayList<>();
        for (DataSensitiveWordsPo dataSensitiveWordsPo:dataSensitiveWordsPos
                ) {
            for (Integer id:wordIds) {
                if ((dataSensitiveWordsPo.getWordId().equals(id))){
                    names.add(dataSensitiveWordsPo.getName());
                }
            }
        }
        return names.stream().collect(Collectors.joining(","));
    }

    /**
     * 敏感字段关联t_dsp_data_type_word_rel
     * @param wordIds
     * @return
     */
    private List<DataTypeWordRelPo> getDataTypeWordRelPoByWordIds(List<Integer> wordIds){
        List<DataTypeWordRelPo> dataTypeWordRelPo = new ArrayList<>();
        for (Integer w:wordIds
                ) {
            DataTypeWordRelCondition dataTypeWordRelCondition = new DataTypeWordRelCondition();
            dataTypeWordRelCondition.setWordId(w);
            dataTypeWordRelCondition.setPageSize(Integer.MAX_VALUE);
            List<DataTypeWordRelPo> dataTypeWordRelPo1 = dataTypeWordRelService.queryList(dataTypeWordRelCondition);
            dataTypeWordRelPo.addAll(dataTypeWordRelPo1);
        }
        return dataTypeWordRelPo;
    }

    /**
     * t_dsp_data_type_word_rel关联t_dsp_data_type
     * @param dataTypeWordRelPos
     * @return
     */
    private List<DataTypePo> getDataTypePoByTypeIds(List<DataTypeWordRelPo> dataTypeWordRelPos){
        List<DataTypePo > dataTypePos = new ArrayList<>();
        for (DataTypeWordRelPo dataTypeWordRelPo:dataTypeWordRelPos
                ) {
            DataTypePo dataTypePo = dataTypeService.query(dataTypeWordRelPo.getTypeId());
            if(dataTypePo!=null){
                dataTypePos.add(dataTypePo);
            }
        }
        return dataTypePos;
    }

    /**
     * 敏感字段关联t_dsp_data_type_word_rel
     * @param wordId
     * @return
     */
    private List<DataTypeWordRelPo> getDataTypeWordRelPoByWordIds(Integer wordId){
        DataTypeWordRelCondition dataTypeWordRelCondition = new DataTypeWordRelCondition();
        dataTypeWordRelCondition.setWordId(wordId);
        List<DataTypeWordRelPo> dataTypeWordRelPo = dataTypeWordRelService.queryList(dataTypeWordRelCondition);
        return dataTypeWordRelPo;
    }

    /**
     * t_dsp_data_level_type_rel关联t_dsp_data_level
     * @param dataLevelTypeRelPos
     * @return
     */
    private List<DataLevelPo> getDataLevelPoByLevelIds(List<DataLevelTypeRelPo> dataLevelTypeRelPos){
        List<DataLevelPo > dataLevelPos = new ArrayList<>();
        for (DataLevelTypeRelPo dataLevelTypeRelPo:dataLevelTypeRelPos
                ) {
            DataLevelPo dataLevelPo = dataLevelService.query(dataLevelTypeRelPo.getLevelId());
            if(dataLevelPo!=null){
                dataLevelPos.add(dataLevelPo);
            }
        }
        return dataLevelPos;
    }

    @Override
    public List<DbScanResultPo>  queryTableeColumnList(DbScanResultCondition dbScanResultCondition){
        PageHelperUtil.startPage(dbScanResultCondition.getPageNum(),dbScanResultCondition.getPageSize());
        return dbScanResultMapper.selectTableColumnList(dbScanResultCondition);
    }
    @Override
    public List<DbScanResultPo> queryDistinctTableList(DbScanResultCondition dbScanResultCondition){
//        PageHelperUtil.startPage(dbScanResultCondition.getPageNum(),dbScanResultCondition.getPageSize());
        return dbScanResultMapper.selectDistinctTableList(dbScanResultCondition);
    }
    @Override
    public Integer countDistinctTable(DbScanResultCondition dbScanResultCondition){
        return dbScanResultMapper.countDistinctTable(dbScanResultCondition);
    }
    @Override
    public Integer countSensColumn(DbScanResultCondition dbScanResultCondition){
        return dbScanResultMapper.countSensColumn(dbScanResultCondition);
    }

    @Override
    public List<DbScanResultPo> queryDbSrPageList(DbScanResultCondition condition) {
        PageHelperUtil.startPage(condition.getPageNum(),condition.getPageSize());
        return dbScanResultMapper.queryDbSrPageList(condition);
    }



    @Override
    public List<DbScanResultPo> queryApproveList(DbScanResultCondition condition) {
        return dbScanResultMapper.queryApproveList(condition);
    }


    private Boolean  isTelephoneNum(String data){
        //正则
        boolean rexFlag = false;
        Pattern pattern = Pattern.compile(TELEPHONENUM_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        return rexFlag;
    }

    /**
     * 固定电话
     * @param data
     * @return
     */
    private  Boolean  isPhoneNum(String data){
        //正则
        boolean rexFlag = false, flag=false;

        Pattern pattern = Pattern.compile(PHONENUM_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
            flag= true;
        }

        //如果正则验证成功，咋验证是否是4位一下
        if(flag){
            String[] strs = data.split("-");
            //如果截取的-长度大于2则不符合规则
            if(strs.length > 2){
                return false;
            }
            int intIndex = data.indexOf("-");
            if(intIndex > 4){
                return false;
            }

            /**
             * 验证是否数字
             */
            for (int i = 0; i < strs.length; i++) {
                Pattern pattern1 = Pattern.compile(NUMBER_REX);
                if(!pattern1.matcher(strs[i]).matches()){
                    return false;
                }
            }
        }


        return rexFlag;
    }

    private Boolean  isDriveLicence(String data,String columnName,String columnRemark){
//        正则+列名-contains[car,driv]or备注contain[驾照,驾驶证]
        boolean rexFlag = false,columnNameFlag = false,columnRemarkFlag = false;


        Pattern pattern = Pattern.compile(DRIVELICENCE_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }


        for (String s:DRIVELICENCE_COLUMNNAME
                ) {
            if(columnName.contains(s)){
                columnNameFlag=true;
                break;
            }
        }
        for (String s:DRIVELICENCE_COLUMNNREMARK
                ) {
            if(columnRemark!=null && columnRemark.toLowerCase().contains(s)){
                columnRemarkFlag=true;
                break;
            }
        }
        return rexFlag&&(columnNameFlag||columnRemarkFlag);
    }
    private Boolean  isBankAccount(String data){


        //正则， /^([1-9]{1})(\d{14}|\d{18})$/; 我们不需要知道属于哪个具体的银行，正则就可以了
        Boolean rexFlag = false;
        if (StringUtils.isEmpty(data) || !data.startsWith("62")) {
            return false;
        }
        Pattern pattern = Pattern.compile(BANKACCOUNT_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        return rexFlag;
    }
    private Boolean  isEmail(String data){
        //正则
        boolean rexFlag = false;
        Pattern pattern = Pattern.compile(EMAIL_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        return rexFlag;
    }
    private Boolean  isQQNum(String data,String columnName,String columnRemark){
        //正则
        boolean rexFlag = false,columnNameFlag = false,columnRemarkFlag = false;
        Pattern pattern = Pattern.compile(QQNUM_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        for (String s:QQNUM_COLUMNNAME
                ) {
            if(columnName.contains(s)){
                columnNameFlag=true;
                break;
            }
        }
        for (String s:QQNUM_COLUMNNREMARK
                ) {
            if(columnRemark!=null && columnRemark.toLowerCase().contains(s)){
                columnRemarkFlag=true;
                break;
            }
        }
        return rexFlag&&(columnNameFlag||columnRemarkFlag);
    }
    private Boolean  isOccupation(String data){
        //关键字，结尾是列表之一[医生，教师，工程师,公务员，警察，交警....]
        boolean rexFlag = false;
        for (String s:OCCUPATION
             ) {
            if(data.endsWith(s)){
                rexFlag=true;
                break;
            }
        }
        return rexFlag;
    }
    private Boolean  isWorkPlace(String data){
        //关键字，结尾是列表之一[公司,单位，医院，]
        boolean rexFlag = false;
        for (String s:WORKPLACE
                ) {
            if(data.endsWith(s)){
                rexFlag=true;
                break;
            }
        }
        return rexFlag;
    }
    private Boolean  isBirthday(String data,String columnName,String columnRemark){
    //列名-contains(born,birdth)or 备注contains[出生]  and value is 日期（正则）
        boolean flag = false,columnNameFlag = false,columnRemarkFlag = false;
//        Pattern pattern = Pattern.compile(DRIVELICENCE_REX);
//        Matcher matcher = pattern.matcher(data);
//        while (matcher.find()) {
//            wordId = true;
//        }
        if(DateUtil.isValidDate(data)){
            for (String s:BIRTHDAY_COLUMNNAME
                    ) {
                if(columnName.contains(s)){
                    columnNameFlag=true;
                    return columnNameFlag;
                }
            }
            for (String s:BIRTHDAY_COLUMNNREMARK
                    ) {
                if(columnRemark!=null && columnRemark.toLowerCase().contains(s)){
                    columnRemarkFlag=true;
                    return columnRemarkFlag;
                }
            }
        }
        return flag;
    }
    private Boolean  isGender(String data,String columnName,String columnRemark){
        //    列名-contains(sex)or 备注contains[性别]  and value is [0,1,f,m,F,M]
        boolean flag = false,columnNameFlag = false,columnRemarkFlag = false;
        for (String s:GENDER
        ) {
            if(data.contains(s)){
                flag=true;
                break;
            }
        }
        for (String s:GENDER_COLUMNNAME
        ) {
            if(columnName.contains(s)){
                columnNameFlag=true;
                return columnNameFlag;
            }
        }
        for (String s:GENDER_COLUMNNREMARK
        ) {
            if(columnRemark!=null && columnRemark.toLowerCase().contains(s)){
                columnRemarkFlag=true;
                return columnRemarkFlag;
            }
        }
        return flag&&(columnNameFlag||columnRemarkFlag);
    }

    /**
     * 籍贯
     * @param data
     * @return
     */
    private Boolean  isHometown(String data){
        //value is [江苏(省)，安徽(省)..33个省]，[南京(市)，苏州(市)等几百个市]
        boolean flag = false;

        if (StringUtils.isEmpty(data) || !data.endsWith("市")) {
            return false;
        }

        for (String[] place:HOMETOWN
                ) {
            for (String s:place
                 ) {
                if(data.contains(s)){
                    flag=true;
                    break;
                }
            }
        }

        return flag;
    }
    private Boolean  isPassword(String data,String columnName,String columnRemark){
        // 列名contains[pwd,password]or备注 contains[密码] and value 拥有10个以上的英文字符
        Boolean wordId = false;
        if(StringUtil.enCharacter(data)>=10){
            for (String s:PASSWORD_COLUMNNAME
                    ) {
                if(columnName.contains(s)){
                    wordId=true;
                    return wordId;
                }
            }
            for (String s:PASSWORD_COLUMNNREMARK
                    ) {
                if(columnRemark!=null && columnRemark.toLowerCase().contains(s)){
                    wordId=true;
                    return wordId;
                }
            }
        }
        return wordId;
    }
    private Boolean  isMac(String data){
        //正则
        Boolean rexFlag = false;
        Pattern pattern = Pattern.compile(MAC_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        return rexFlag;
    }
    private Boolean  isOs(String data){
        //value is [江苏(省)，安徽(省)..33个省]，[南京(市)，苏州(市)等几百个市]
        Boolean flag = false;
        for (String s:OS
                ) {
            if(data.contains(s)){
                flag=true;
                break;
            }
        }
        return flag;
    }
    private Boolean  isName(String data){
        //正则
        boolean rexFlag = false;
        Pattern pattern = Pattern.compile(NAME_REX);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        return rexFlag;
    }
    private Boolean  isIdNum(String data,String columnName,String columnRemark){
        boolean rexFlag = false, columnNameFlag = false, columnRemarkFlag = false;
        Pattern pattern = Pattern.compile(DRIVELICENCE_REX);
        Matcher matcher = pattern.matcher(data);
        if (StringUtils.isEmpty(data) || data.length() != 18) {
            return false;
        }
        while (matcher.find()) {
            rexFlag = true;
        }

    /*    while (matcher.find()) {
            rexFlag = true;
        }
        for (String s:DRIVELICENCE_COLUMNNAME
        ) {
            if(columnName.contains(s)){
                columnNameFlag=true;
                break;
            }
        }
        for (String s:DRIVELICENCE_COLUMNNREMARK
        ) {
            if(columnRemark!=null && columnRemark.toLowerCase().contains(s)){
                columnRemarkFlag=true;
                break;
            }
        }*/
        return rexFlag&&!(columnNameFlag||columnRemarkFlag);
    }

    /**
     * 正则表达式验证地址
     * @param data
     * @return
     *
     */
    private  Boolean  isAddress (String data){

        boolean flag = false;
        for (String[] place:HOMETOWN
        ) {
            for (String s:place
            ) {
                if(data.contains(s)){
                    flag=true;
                    break;
                }
            }
        }
        boolean rexFlag = false;
        Pattern pattern = Pattern.compile(ADDRESS_REX_MATCHER);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }


        return flag&&rexFlag;
    }

    private Boolean matchString (String matchString,String data,String method){
        if("contains".equalsIgnoreCase(method)){
            for (String s:matchString.split(",")
            ) {
                if(data.contains(s)){
                    return true;
                }
            }
        }
        else if("endsWith".equalsIgnoreCase(method)){
            for (String s:matchString.split(",")
            ) {
                if(data.endsWith(s)){
                    return true;
                }
            }
        }
        else {
            for (String s:matchString.split(",")
            ) {
                if(data.equals(s)){
                    return true;
                }
            }
        }
        return false;
    }

    private Boolean matchLength (String length,Integer valueLength){
        for (String s:length.split(",")
        ) {
            if(valueLength.equals(Integer.valueOf(s))){
                return true;
            }
        }
        return false;
    }

    private Boolean matchRex(String rex,String data){
        boolean rexFlag = false;
        Pattern pattern = Pattern.compile(rex);
        Matcher matcher = pattern.matcher(data);
        while (matcher.find()) {
            rexFlag = true;
        }
        return rexFlag;
    }
}